"""
清洗从淘宝外购的tick数据
"""
import os
import pandas as pd
from gateway_mapping import exchange_mapping
from datetime import datetime, timedelta

# 手工参数
trading_date = "20231226"
trading_date_delta = 1  # 夜盘日期与交易日的天数差，周末是3，平时是1

#+++++++++
ori_tick_data_folder = f"E:\外购tick\marketdatacsv{trading_date}"
new_folder = f"E:\\外购tick\\{trading_date}"
if not os.path.exists(new_folder):
    os.mkdir(new_folder)

columns_name = ["trading_date",
                "symbol",
                "dt",
                "ms",
                "last_price",
                "volume",
                "bid_price_1",
                "bid_volume_1",
                "ask_price_1",
                "ask_volume_1",
                "AveragePrice",
                "turnover",
                "open_interest",
                "limit_up",
                "limit_down"]

# 第一步，清除多余数文件
all_tick_files = os.listdir(ori_tick_data_folder)
for tick_file in all_tick_files:
    if "-" in tick_file:
        print(tick_file)
        os.remove(f"{ori_tick_data_folder}\\{tick_file}")
    elif len(tick_file) > 11 and " " not in tick_file:
        print(tick_file)
        os.remove(f"{ori_tick_data_folder}\\{tick_file}")
    elif "efp" in tick_file:
        print(tick_file)
        os.remove(f"{ori_tick_data_folder}\\{tick_file}")
    """
    if "SPC " in tick_file or "IPS " in tick_file:
        print(tick_file)
        os.remove(f"{ori_tick_data_folder}\\{tick_file}")
    """
all_tick_files = os.listdir(ori_tick_data_folder)
for file_name in all_tick_files:
    print(file_name)
    symbol, _ = file_name.split(".")
    if " " in file_name:
        _, fn = file_name.split(" ")
        sec_id = fn[:2] if fn[1].isalpha() else fn[0]
    else:
        sec_id = symbol[:2] if symbol[1].isalpha() else symbol[0]
    exchange = exchange_mapping[sec_id]
    exchange_symbol = f"Exchange.{exchange}"
    vt_symbol = f"{symbol}.{exchange}"
    df = pd.read_csv(f"{ori_tick_data_folder}\\{file_name}", names=columns_name)
    if df.shape[0] < 2:
        continue
    df["exchange"] = exchange_symbol
    df["pre_close"] = df.iloc[0]["last_price"]
    df["open_price"] = df.iloc[1]["last_price"]
    df["high_price"] = df["last_price"].expanding().max()
    df["low_price"] = df["last_price"].expanding().min()
    for p in [2, 3, 4, 5]:
        df[f"bid_price_{p}"] = 0
        df[f"ask_price_{p}"] = 0
        df[f"bid_volume_{p}"] = 0
        df[f"ask_volume_{p}"] = 0
    df["vt_symbol"] = vt_symbol
    df["sec_id"] = sec_id
    df["name"] = symbol
    df["gateway_name"] = "CTP_main"

    df["trading_date"] = df["trading_date"].astype(str)
    df = df.loc[df["trading_date"] == trading_date]   # 有些上一个交易日的数据要过滤掉
    df["ms"] = df["ms"].astype(str)

    df["datetime"] = df["trading_date"].str.cat(df["dt"], sep=" ")
    df["datetime"] = df["datetime"].str.cat(df["ms"], sep=".")
    df["datetime"] = pd.to_datetime(df["datetime"], format="%Y%m%d %H:%M:%S.%f")

    df["trading"] = False
    # 调整日盘的trading标志
    df.loc[df["datetime"].dt.hour == 9, "trading"] = True
    df.loc[(df["datetime"].dt.hour == 10) & (df["datetime"].dt.minute < 15), "trading"] = True
    df.loc[(df["datetime"].dt.hour == 10) & (df["datetime"].dt.minute >= 30), "trading"] = True
    df.loc[(df["datetime"].dt.hour == 11) & (df["datetime"].dt.minute < 30), "trading"] = True
    df.loc[(df["datetime"].dt.hour == 13) & (df["datetime"].dt.minute >= 30), "trading"] = True
    df.loc[df["datetime"].dt.hour == 14, "trading"] = True

    # 调整夜盘的trading标志
    hours = set(df["datetime"].dt.hour.to_list())
    night_trade = True
    if 2 in hours:
        # 2:30
        df.loc[df["datetime"].dt.hour >= 21, "trading"] = True
        df.loc[df["datetime"].dt.hour < 2, "trading"] = True
        df.loc[(df["datetime"].dt.hour == 2) & (df["datetime"].dt.minute < 30), "trading"] = True
        pass
    elif 0 in hours:
        # 01:00
        df.loc[df["datetime"].dt.hour >= 21, "trading"] = True
        df.loc[df["datetime"].dt.hour < 1, "trading"] = True
        pass
    elif 22 in hours:
        # 23:00
        df.loc[df["datetime"].dt.hour == 21, "trading"] = True
        df.loc[df["datetime"].dt.hour == 22, "trading"] = True
        pass
    else:
        # no night trade
        night_trade = False
    # 日期调整
    if night_trade:
        # 对于时间在8点之前，以及20点之后的，日期填入上一个交易日
        df.loc[df["datetime"].dt.hour >= 20, "datetime"] -= timedelta(days=trading_date_delta)
        if trading_date_delta != 1:
            df.loc[df["datetime"].dt.hour < 8, "datetime"] -= timedelta(days=trading_date_delta)
            df.loc[df["datetime"].dt.hour < 8, "datetime"] += timedelta(days=1)

    df["local_time"] = df["datetime"]
    df.drop(["dt", "ms", "AveragePrice"], axis=1, inplace=True)
    df.to_csv(f"{new_folder}\\{trading_date} tick_{vt_symbol}.csv", index=False)